# -*- coding: utf-8 -*-
import scrapy
import json
from ..items import MiaoItem
import random


class BaiduSpider(scrapy.Spider):
    page = 1
    name = 'baidu'
    allowed_domains = ['image.baidu.com']
    color = 1
    start_urls = [
        'https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&fp=result&queryWord=美女&cl=2&lm=-1&ie=utf-8&oe=utf-8&word=美女&ic=8&pn=30']
    custom_settings = {'DOWNLOAD_DELAY': 0.2, 'CONCURRENT_REQUESTS_PER_IP': 4, 'DOWNLOADER_MIDDLEWARES': {},
                       'DEFAULT_REQUEST_HEADERS': {
                           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36',
                           'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
                           'Accept-Language': 'en',
                       }}

    def parse(self, response):
        imgAll = json.loads(response.body.decode('utf-8'))
        is_open = 0
        num = 0
        length = len(imgAll['data'])
        for img in imgAll['data']:
            num += 1
            if num == length - 1:
                is_open = 1
                self.page += 1
            # 使用在items.py创建的数据模型item
            item = MiaoItem()
            imgurl = img['middleURL']
            # print(imgurl)
            item['src'] = [imgurl]
            yield item
            if is_open:
                url = 'https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&fp=result&queryWord=美女&cl=2&lm=-1&ie=utf-8&oe=utf-8&ic=8&word=美女&pn={}'.format(
                    self.page * 30)
                #     # 将url传给scrapy.Request 得到的结果继续用self.parse处理
                yield scrapy.Request(url=url, callback=self.parse)
        pass

# class RandomUserAgentMiddleware(object):
#     def process_request(self, request, spider):
#         rand_use = random.choice(USER_AGENT_LIST)
#         if rand_use:
#             request.headers.setdefault('User-Agent', rand_use)
